package 月度_季度_年度数据;

import Moudle.Excel_class;
import Moudle.Item;
import Util.HttpConnect;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.ProtocolException;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.*;

public class Main {
    /*
    * 运行时 只需根据需要 更换time_code的值
    * 抓取一段时间后 网站会禁止你的访问 需等待 10~20分钟 再次运行
    * */


    /*
    * itemMap：包含所有已经抓取的信息
    * localFile：本地已存在的文件
    * */
    static Map<String, Item> itemMap = new HashMap<>();
    static Set<String> localFile;

    static String time_code;
    static Log LOG = LogFactory.getLog(Main.class);

    public static void main(String[] args) throws InterruptedException {
       // time_code = "hgyd";//hgyd 月度数据  hgjd 季度数据    hgnd年度数据
        //START();
        time_code = "hgnd";
        START();
    }

    static void START() throws InterruptedException {
        try {
            String path = null;

            switch (time_code){
                case "hgyd":
                    path = "D:\\测试临时文件\\国家统计局-月度数据";
                    break;
                case "hgjd":
                    path = "D:\\测试临时文件\\国家统计局-季度数据";
                    break;
                case "hgnd":
                    path = "D:\\测试临时文件\\国家统计局-年度数据";
                    break;
            }

            localFile = getAllFile(path, false);
            TraverseNode("zb");

        }catch (Exception e){
            e.printStackTrace();

            if (e instanceof CantVisitTheWePageException){
                System.out.println("---休眠二十分钟----");
                Thread.sleep(1200000);
                START();
            }
        }
    }

    static void TraverseNode(String id) throws CantVisitTheWePageException, NullPointerException, ProtocolException, InterruptedException {
        String url = "https://data.stats.gov.cn/easyquery.htm";

        Map<String, String> header_map = new HashMap<>();
        header_map.put("Content-Type", "application/x-www-form-urlencoded");

        Map<String, String> data_map = new HashMap<>();
        data_map.put("id", id);
        data_map.put("dbcode", time_code); //hgyd 月度数据  hgjd 季度数据    hgnd年度数据
        data_map.put("wdcode", "zb");
        data_map.put("m", "getTree");

        Thread.sleep(1000);
        LOG.info("POST id =" + id);
        String jsonStr = HttpConnect.HttpPost_OkHttp(url, header_map, data_map);

        if (jsonStr == null) {
            throw new NullPointerException(jsonStr);
        }

        if (!Can_VisitTheWebPage(jsonStr)) //无法访问网页一段事时间后 再次运行
            throw new CantVisitTheWePageException(jsonStr);

        Item[] items = ParseData.parseJsonStrToItem(jsonStr);
        for (Item item : items){
            itemMap.put(item.id, item);
            if(item.isParent.equals("true")){
                TraverseNode(item.id);
            }else{//叶节点
                if (!localFile.contains(item.name)) //已在本地 不再抓取
                    getData(item);
            }
        }
    }

    //所有数据都在叶节点
    static void getData(Item item) throws CantVisitTheWePageException {
        String time = "2019,2020";
        String source_encode = "[{\"wdcode\":\"zb\",\"valuecode\":\""+ item.id +"\"},{\"wdcode\":\"sj\",\"valuecode\":\""+ time +"\"}]";
        try {
            String encode = URLEncoder.encode(source_encode, "UTF-8");
            String url = "https://data.stats.gov.cn/easyquery.htm?m=QueryData&dbcode="+ item.dbcode +"&rowcode=zb&colcode=sj&wds=%5B%5D&dfwds="+encode;

            Thread.sleep(1000);
            LOG.info("GET item.name =" + item.name);
            String jsonStr = HttpConnect.HttpGet_OkHttp(url, null);

            if (!Can_VisitTheWebPage(jsonStr)) //无法访问网页一段事时间后 再次运行
                throw new CantVisitTheWePageException(item.name);
            Excel_class[] excel_classes = ParseData.parseData(jsonStr);
            ToExcel.toExcel(excel_classes, item, time_code);

            //请求太快 网站会拒绝访问 在这停顿2秒
            Thread.sleep(2000);

        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
            System.out.println("-------------encode_err------------\n\n");
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    public static Set<String> getAllFile(String directoryPath, boolean isAddDirectory) {
        //获取本地所有文件
        Set<String> list = new HashSet<>();
        File baseFile = new File(directoryPath);
        if (baseFile.isFile() || !baseFile.exists()) {
            return list;
        }
        File[] files = baseFile.listFiles();
        for (File file : files) {
            if (file.isDirectory()) {
                if(isAddDirectory){
                    list.add(file.getAbsolutePath());
                }
                list.addAll(getAllFile(file.getAbsolutePath(),isAddDirectory));
            } else {
                String fileName = file.getName();
                list.add(fileName.substring(0, fileName.indexOf(".")));
            }
        }
        return list;
    }


    static boolean Can_VisitTheWebPage(String str){
        //网站有反爬虫 多次请求会拒绝访问
        if (str == null || str.contains("请开启JavaScript并刷新该页") || str.contains("307 Temporary Redirect")){
            return false;
        }
        return true;
    }



}


